In [1]:
from tensorflow.keras import backend as K
import gc
K.clear_session()
gc.collect()
WARNING:tensorflow:From C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\backend\common\global_state.py:82: The name tf.reset_default_graph is deprecated. Please use tf.compat.v1.reset_default_graph instead.
Out[1]:
0
Multiclass classification¶
Step1: Data Preparation¶
In [2]:
import numpy as np
import matplotlib.pyplot as plt
import h5py
import tensorflow as tf
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.keras.utils import to_categorical
from sklearn.utils import class_weight
from tensorflow.keras.metrics import Precision, Recall, AUC
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense, Dropout
In [3]:
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
In [4]:
# Load the HDF5 file
bird_sound = h5py.File("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\\Homework3\\bird_spectrograms.hdf5", 'r')
In [5]:
species= list(bird_sound.keys())
# Load all spectrograms and labels
X = []
y = []
for label in species:
data =bird_sound[label][:]
data = np.transpose(data, (2, 0, 1))
X.append(data)
y += [label] * data.shape[0]
X = np.concatenate(X, axis=0)
print(X.shape)
(1981, 128, 517)
In [6]:
# Plot one spectrogram sample from each species
unique_species = list(np.unique(y))
sample_specs = {}
for label in unique_species:
label_index = y.index(label) if isinstance(y, list) else np.where(np.array(y) == label)[0][0]
sample = X[label_index, :, :]
sample_specs[label] = sample
plt.figure(figsize=(15, 3 * len(sample_specs)))
for i, (label, spec) in enumerate(sample_specs.items()):
plt.subplot(len(sample_specs), 1, i + 1)
plt.imshow(spec, aspect='auto', origin='lower', cmap='inferno')
plt.title(f"Species: {label}")
plt.colorbar()
plt.axis('off')
plt.tight_layout()
plt.show()
Step2: Train-Test Split¶
In [7]:
X = X.reshape((1981, 128, 517, 1))
y = np.array(y)
label_encoder = LabelEncoder()
y_encoded = label_encoder.fit_transform(y)
y_categorical = to_categorical(y_encoded)
# Train-test split
X_train, X_test, y_train, y_test = train_test_split(
X, y_categorical, test_size=0.2, random_state=156, stratify=y_encoded
)
print(f"X_train shape: {X_train.shape}, y_train shape: {y_train.shape}")
print(f"X_test shape: {X_test.shape}, y_test shape: {y_test.shape}")
print(f"Number of classes: {len(label_encoder.classes_)}")
X_train shape: (1584, 128, 517, 1), y_train shape: (1584, 12) X_test shape: (397, 128, 517, 1), y_test shape: (397, 12) Number of classes: 12
Step3: Model Training¶
Model1¶
In [8]:
# Definigng model1
np.random.seed(123)
tf.random.set_seed(123)
model1_multi = Sequential()
model1_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1_multi.add(MaxPooling2D(pool_size=(2, 2)))
model1_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1_multi.add(MaxPooling2D(pool_size=(2, 2)))
model1_multi.add(Flatten())
model1_multi.add(Dense(128, activation='relu'))
model1_multi.add(Dropout(0.5))
model1_multi.add(Dense(12, activation='softmax'))
model1_multi.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc')
]
)
C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [9]:
import time
start_time = time.time()
history1 = model1_multi.fit(
X_train, y_train,
validation_data=(X_test, y_test),
epochs=20,
batch_size=256
)
end_time = time.time()
time1 = end_time - start_time
print(f"\nTraining time: {time1:.2f} seconds")
Epoch 1/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.1411 - auc: 0.5307 - loss: 545.9827 - precision: 0.1426 - recall: 0.1411 - val_accuracy: 0.0453 - val_auc: 0.4780 - val_loss: 59.9108 - val_precision: 0.0455 - val_recall: 0.0453 Epoch 2/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.1031 - auc: 0.5173 - loss: 44.2694 - precision: 0.1022 - recall: 0.0866 - val_accuracy: 0.3073 - val_auc: 0.5002 - val_loss: 2.4888 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 3/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.2602 - auc: 0.5037 - loss: 3.2944 - precision: 0.0763 - recall: 0.0186 - val_accuracy: 0.0982 - val_auc: 0.4269 - val_loss: 2.9123 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 4/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.1834 - auc: 0.5233 - loss: 2.7006 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.3476 - val_auc: 0.6964 - val_loss: 2.4225 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 5/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.3829 - auc: 0.7179 - loss: 2.3431 - precision: 0.3555 - recall: 0.0182 - val_accuracy: 0.3602 - val_auc: 0.7486 - val_loss: 2.2619 - val_precision: 1.0000 - val_recall: 0.0025 Epoch 6/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.4012 - auc: 0.7967 - loss: 2.0732 - precision: 0.8099 - recall: 0.0506 - val_accuracy: 0.4030 - val_auc: 0.7713 - val_loss: 2.0368 - val_precision: 0.5932 - val_recall: 0.0882 Epoch 7/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.4880 - auc: 0.8424 - loss: 1.7971 - precision: 0.6669 - recall: 0.1303 - val_accuracy: 0.4081 - val_auc: 0.7851 - val_loss: 2.0887 - val_precision: 0.5025 - val_recall: 0.2569 Epoch 8/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 4s/step - accuracy: 0.5516 - auc: 0.8821 - loss: 1.5779 - precision: 0.7027 - recall: 0.2674 - val_accuracy: 0.4131 - val_auc: 0.7977 - val_loss: 1.9996 - val_precision: 0.5183 - val_recall: 0.2141 Epoch 9/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 33s 5s/step - accuracy: 0.6463 - auc: 0.9291 - loss: 1.2091 - precision: 0.7768 - recall: 0.4323 - val_accuracy: 0.3703 - val_auc: 0.7754 - val_loss: 2.2326 - val_precision: 0.4770 - val_recall: 0.2091 Epoch 10/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 34s 5s/step - accuracy: 0.7044 - auc: 0.9524 - loss: 1.0085 - precision: 0.8508 - recall: 0.4959 - val_accuracy: 0.4257 - val_auc: 0.8066 - val_loss: 2.0428 - val_precision: 0.5694 - val_recall: 0.2997 Epoch 11/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 33s 5s/step - accuracy: 0.7876 - auc: 0.9773 - loss: 0.7180 - precision: 0.9011 - recall: 0.6416 - val_accuracy: 0.4408 - val_auc: 0.8011 - val_loss: 2.0789 - val_precision: 0.5631 - val_recall: 0.2922 Epoch 12/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.8431 - auc: 0.9899 - loss: 0.5213 - precision: 0.9326 - recall: 0.7390 - val_accuracy: 0.4332 - val_auc: 0.8012 - val_loss: 2.1565 - val_precision: 0.5402 - val_recall: 0.3048 Epoch 13/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.9085 - auc: 0.9956 - loss: 0.3503 - precision: 0.9638 - recall: 0.8365 - val_accuracy: 0.4559 - val_auc: 0.7993 - val_loss: 2.3629 - val_precision: 0.5349 - val_recall: 0.3476 Epoch 14/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9285 - auc: 0.9976 - loss: 0.2582 - precision: 0.9752 - recall: 0.8818 - val_accuracy: 0.4307 - val_auc: 0.8021 - val_loss: 2.4970 - val_precision: 0.5068 - val_recall: 0.3778 Epoch 15/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 32s 5s/step - accuracy: 0.9570 - auc: 0.9992 - loss: 0.1659 - precision: 0.9755 - recall: 0.9340 - val_accuracy: 0.4408 - val_auc: 0.8051 - val_loss: 2.5254 - val_precision: 0.4765 - val_recall: 0.3577 Epoch 16/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9673 - auc: 0.9994 - loss: 0.1530 - precision: 0.9851 - recall: 0.9394 - val_accuracy: 0.4257 - val_auc: 0.7952 - val_loss: 2.8406 - val_precision: 0.4763 - val_recall: 0.3804 Epoch 17/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9562 - auc: 0.9993 - loss: 0.1310 - precision: 0.9726 - recall: 0.9422 - val_accuracy: 0.4710 - val_auc: 0.8041 - val_loss: 2.5447 - val_precision: 0.5288 - val_recall: 0.3929 Epoch 18/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9813 - auc: 0.9996 - loss: 0.0987 - precision: 0.9884 - recall: 0.9640 - val_accuracy: 0.4332 - val_auc: 0.8000 - val_loss: 2.6519 - val_precision: 0.4901 - val_recall: 0.3753 Epoch 19/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9797 - auc: 0.9997 - loss: 0.0756 - precision: 0.9891 - recall: 0.9743 - val_accuracy: 0.4257 - val_auc: 0.7952 - val_loss: 2.9259 - val_precision: 0.4985 - val_recall: 0.4106 Epoch 20/20 7/7 ━━━━━━━━━━━━━━━━━━━━ 31s 4s/step - accuracy: 0.9854 - auc: 0.9999 - loss: 0.0571 - precision: 0.9896 - recall: 0.9803 - val_accuracy: 0.4332 - val_auc: 0.7920 - val_loss: 2.7538 - val_precision: 0.4903 - val_recall: 0.3829 Training time: 641.04 seconds
In [10]:
print("\nValidation Accuracy Model1:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
print(f"{m}: {history1.history[m][-1]:.4f}")
Validation Accuracy Model1: val_accuracy: 0.4332 val_precision: 0.4903 val_recall: 0.3829 val_auc: 0.7920
- Model 1 has validation accuracy of (43.32%) and a strong AUC of 0.7844. These results indicate effective overall classification across multiple bird species.
In [11]:
# plotting the results
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(history1.history['accuracy'], label='Train Accuracy')
plt.plot(history1.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(history1.history['loss'], label='Train Loss')
plt.plot(history1.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.tight_layout()
plt.show()
- The graphs show that the model's training accuracy keeps improving, but the validation accuracy stays around 45%, which means the model might be overfitting.
- The loss graph also shows training loss goes down , while validation loss remains mostly flat indicatingthat the model is learning the training data well but not generalizing to new data.
In [12]:
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt
y_pred = model1_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)
cm1 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm1, annot=True, fmt='d', cmap='Blues',
xticklabels=label_encoder.classes_,
yticklabels=label_encoder.classes_)
plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()
13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 117ms/step
- This confusion matrix shows that the model performs best on houspa (House Sparrow), predicting it correctly 106 times. However, it struggles to distinguish between several other bird species like bewwre, rewbla, and sonspa, often confusing them with each other.
Model2¶
In [13]:
# Defining model2
np.random.seed(42)
tf.random.set_seed(42)
model2_multi = Sequential()
model2_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2_multi.add(MaxPooling2D(pool_size=(2, 2)))
model2_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2_multi.add(MaxPooling2D(pool_size=(2, 2)))
model2_multi.add(Flatten())
model2_multi.add(Dense(64, activation='relu'))
model2_multi.add(Dropout(0.3))
model2_multi.add(Dense(12, activation='softmax'))
model2_multi.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc')
]
)
C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [14]:
# Tracking the model training time
import time
start_time = time.time()
history2 = model2_multi.fit(
X_train, y_train,
validation_data=(X_test, y_test),
epochs=15,
batch_size=128
)
end_time = time.time()
time2 = end_time - start_time
print(f"\nTraining time: {time2:.2f} seconds")
Epoch 1/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 11s 694ms/step - accuracy: 0.1675 - auc: 0.5497 - loss: 225.6911 - precision: 0.1647 - recall: 0.1556 - val_accuracy: 0.3174 - val_auc: 0.6720 - val_loss: 2.3636 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 2/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 654ms/step - accuracy: 0.1788 - auc: 0.6281 - loss: 2.4847 - precision: 0.0977 - recall: 0.0025 - val_accuracy: 0.2771 - val_auc: 0.6823 - val_loss: 2.3905 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 3/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 670ms/step - accuracy: 0.2556 - auc: 0.6828 - loss: 2.4334 - precision: 0.3094 - recall: 0.0339 - val_accuracy: 0.3149 - val_auc: 0.7282 - val_loss: 2.2707 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 4/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 681ms/step - accuracy: 0.3738 - auc: 0.7624 - loss: 2.1663 - precision: 0.5728 - recall: 0.0493 - val_accuracy: 0.3778 - val_auc: 0.7536 - val_loss: 2.1355 - val_precision: 0.3929 - val_recall: 0.0554 Epoch 5/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 676ms/step - accuracy: 0.5174 - auc: 0.8676 - loss: 1.7304 - precision: 0.6833 - recall: 0.1256 - val_accuracy: 0.4081 - val_auc: 0.7968 - val_loss: 1.9734 - val_precision: 0.5104 - val_recall: 0.1234 Epoch 6/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 668ms/step - accuracy: 0.6408 - auc: 0.9278 - loss: 1.2562 - precision: 0.8154 - recall: 0.3680 - val_accuracy: 0.4131 - val_auc: 0.7979 - val_loss: 2.0009 - val_precision: 0.5152 - val_recall: 0.1285 Epoch 7/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 661ms/step - accuracy: 0.7841 - auc: 0.9761 - loss: 0.7922 - precision: 0.9134 - recall: 0.5615 - val_accuracy: 0.4307 - val_auc: 0.7871 - val_loss: 2.3146 - val_precision: 0.4957 - val_recall: 0.2897 Epoch 8/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 653ms/step - accuracy: 0.8689 - auc: 0.9894 - loss: 0.4926 - precision: 0.9371 - recall: 0.7634 - val_accuracy: 0.4358 - val_auc: 0.8013 - val_loss: 2.2728 - val_precision: 0.4791 - val_recall: 0.3174 Epoch 9/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 661ms/step - accuracy: 0.9118 - auc: 0.9940 - loss: 0.3379 - precision: 0.9584 - recall: 0.8525 - val_accuracy: 0.4282 - val_auc: 0.7932 - val_loss: 2.2156 - val_precision: 0.5138 - val_recall: 0.2821 Epoch 10/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 671ms/step - accuracy: 0.9505 - auc: 0.9960 - loss: 0.2284 - precision: 0.9681 - recall: 0.9132 - val_accuracy: 0.4005 - val_auc: 0.7658 - val_loss: 2.6874 - val_precision: 0.4648 - val_recall: 0.2997 Epoch 11/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 693ms/step - accuracy: 0.9315 - auc: 0.9961 - loss: 0.3092 - precision: 0.9578 - recall: 0.8611 - val_accuracy: 0.4207 - val_auc: 0.7729 - val_loss: 2.5517 - val_precision: 0.4835 - val_recall: 0.3325 Epoch 12/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 686ms/step - accuracy: 0.9635 - auc: 0.9990 - loss: 0.1784 - precision: 0.9795 - recall: 0.9316 - val_accuracy: 0.4106 - val_auc: 0.7864 - val_loss: 2.6806 - val_precision: 0.4804 - val_recall: 0.3401 Epoch 13/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 687ms/step - accuracy: 0.9716 - auc: 0.9991 - loss: 0.1314 - precision: 0.9805 - recall: 0.9521 - val_accuracy: 0.3980 - val_auc: 0.7801 - val_loss: 2.9548 - val_precision: 0.4441 - val_recall: 0.3300 Epoch 14/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 671ms/step - accuracy: 0.9817 - auc: 0.9995 - loss: 0.0800 - precision: 0.9859 - recall: 0.9755 - val_accuracy: 0.4257 - val_auc: 0.7916 - val_loss: 2.8339 - val_precision: 0.5017 - val_recall: 0.3728 Epoch 15/15 13/13 ━━━━━━━━━━━━━━━━━━━━ 9s 668ms/step - accuracy: 0.9873 - auc: 0.9997 - loss: 0.0545 - precision: 0.9915 - recall: 0.9847 - val_accuracy: 0.4635 - val_auc: 0.7810 - val_loss: 3.0003 - val_precision: 0.5081 - val_recall: 0.3929 Training time: 134.07 seconds
In [15]:
print("\nValidation Accuracy Model2:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
print(f"{m}: {history2.history[m][-1]:.4f}")
Validation Accuracy Model2: val_accuracy: 0.4635 val_precision: 0.5081 val_recall: 0.3929 val_auc: 0.7810
- Model 2 has validation accuracy of (46.35%) and a strong AUC of 0.7810. These results shows that it has 3% moere accuracy than model1.
In [16]:
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(history2.history['accuracy'], label='Train Accuracy')
plt.plot(history2.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(history2.history['loss'], label='Train Loss')
plt.plot(history2.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.tight_layout()
plt.show()
In [17]:
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt
y_pred = model2_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)
cm2 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm2, annot=True, fmt='d', cmap='Blues',
xticklabels=label_encoder.classes_,
yticklabels=label_encoder.classes_)
plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()
13/13 ━━━━━━━━━━━━━━━━━━━━ 1s 48ms/step
- The model predicts House Sparrow and song sparrow very accurately, with minimal confusion.
Model1-A¶
In [18]:
np.random.seed(123)
tf.random.set_seed(123)
model1a_multi = Sequential()
model1a_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1a_multi.add(MaxPooling2D(pool_size=(2, 2)))
model1a_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1a_multi.add(MaxPooling2D(pool_size=(2, 2)))
model1a_multi.add(Flatten())
model1a_multi.add(Dense(128, activation='relu'))
model1a_multi.add(Dropout(0.4))
model1a_multi.add(Dense(12, activation='softmax'))
model1a_multi.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc')
]
)
C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [19]:
start_time = time.time()
history3 = model1a_multi.fit(
X_train, y_train,
validation_data=(X_test, y_test),
epochs=20,
batch_size=128
)
end_time = time.time()
time3 = end_time - start_time
print(f"\nTraining time: {time3:.2f} seconds")
Epoch 1/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 2s/step - accuracy: 0.1474 - auc: 0.5388 - loss: 490.2636 - precision: 0.1479 - recall: 0.1436 - val_accuracy: 0.0227 - val_auc: 0.5114 - val_loss: 2.4803 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 2/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.1077 - auc: 0.5389 - loss: 4.6898 - precision: 0.1610 - recall: 0.0580 - val_accuracy: 0.1511 - val_auc: 0.5459 - val_loss: 2.4739 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 3/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.1850 - auc: 0.5440 - loss: 2.4723 - precision: 0.0000e+00 - recall: 0.0000e+00 - val_accuracy: 0.3174 - val_auc: 0.5255 - val_loss: 2.4738 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 4/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.3251 - auc: 0.6218 - loss: 2.4475 - precision: 0.2931 - recall: 0.0156 - val_accuracy: 0.3325 - val_auc: 0.6755 - val_loss: 2.4626 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 5/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.3568 - auc: 0.7039 - loss: 2.4295 - precision: 0.2080 - recall: 0.0055 - val_accuracy: 0.3073 - val_auc: 0.7320 - val_loss: 2.3537 - val_precision: 0.8000 - val_recall: 0.0101 Epoch 6/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.3726 - auc: 0.7713 - loss: 2.2719 - precision: 0.9558 - recall: 0.0228 - val_accuracy: 0.3627 - val_auc: 0.7614 - val_loss: 2.1646 - val_precision: 0.2973 - val_recall: 0.0277 Epoch 7/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.4831 - auc: 0.8300 - loss: 1.8891 - precision: 0.6481 - recall: 0.1299 - val_accuracy: 0.4232 - val_auc: 0.7944 - val_loss: 2.0609 - val_precision: 0.5044 - val_recall: 0.1436 Epoch 8/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.5993 - auc: 0.9144 - loss: 1.3373 - precision: 0.7948 - recall: 0.3693 - val_accuracy: 0.3904 - val_auc: 0.7869 - val_loss: 2.1221 - val_precision: 0.4809 - val_recall: 0.1587 Epoch 9/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.7471 - auc: 0.9612 - loss: 0.9261 - precision: 0.8851 - recall: 0.5404 - val_accuracy: 0.3678 - val_auc: 0.7654 - val_loss: 2.3917 - val_precision: 0.4556 - val_recall: 0.2065 Epoch 10/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.8065 - auc: 0.9766 - loss: 0.7406 - precision: 0.9126 - recall: 0.6287 - val_accuracy: 0.4181 - val_auc: 0.7855 - val_loss: 2.6065 - val_precision: 0.4743 - val_recall: 0.3023 Epoch 11/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.8809 - auc: 0.9901 - loss: 0.4354 - precision: 0.9246 - recall: 0.7986 - val_accuracy: 0.3955 - val_auc: 0.7822 - val_loss: 2.6364 - val_precision: 0.4563 - val_recall: 0.2897 Epoch 12/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.9286 - auc: 0.9970 - loss: 0.2632 - precision: 0.9641 - recall: 0.8865 - val_accuracy: 0.3955 - val_auc: 0.7760 - val_loss: 2.9961 - val_precision: 0.4621 - val_recall: 0.3375 Epoch 13/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 31s 2s/step - accuracy: 0.9571 - auc: 0.9989 - loss: 0.1614 - precision: 0.9749 - recall: 0.9297 - val_accuracy: 0.3955 - val_auc: 0.7766 - val_loss: 3.1419 - val_precision: 0.4400 - val_recall: 0.3325 Epoch 14/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 2s/step - accuracy: 0.9781 - auc: 0.9994 - loss: 0.1087 - precision: 0.9875 - recall: 0.9576 - val_accuracy: 0.3980 - val_auc: 0.7760 - val_loss: 3.2108 - val_precision: 0.4295 - val_recall: 0.3375 Epoch 15/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9758 - auc: 0.9994 - loss: 0.0937 - precision: 0.9864 - recall: 0.9628 - val_accuracy: 0.4106 - val_auc: 0.7697 - val_loss: 3.4532 - val_precision: 0.4303 - val_recall: 0.3501 Epoch 16/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 31s 2s/step - accuracy: 0.9811 - auc: 0.9996 - loss: 0.0762 - precision: 0.9842 - recall: 0.9718 - val_accuracy: 0.3854 - val_auc: 0.7689 - val_loss: 3.5760 - val_precision: 0.4149 - val_recall: 0.3375 Epoch 17/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 32s 3s/step - accuracy: 0.9813 - auc: 0.9991 - loss: 0.0869 - precision: 0.9856 - recall: 0.9717 - val_accuracy: 0.3929 - val_auc: 0.7609 - val_loss: 3.6187 - val_precision: 0.4182 - val_recall: 0.3350 Epoch 18/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9829 - auc: 0.9993 - loss: 0.0803 - precision: 0.9859 - recall: 0.9692 - val_accuracy: 0.3829 - val_auc: 0.7631 - val_loss: 3.6721 - val_precision: 0.4322 - val_recall: 0.3451 Epoch 19/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 34s 3s/step - accuracy: 0.9850 - auc: 0.9996 - loss: 0.0590 - precision: 0.9900 - recall: 0.9770 - val_accuracy: 0.3879 - val_auc: 0.7549 - val_loss: 3.8591 - val_precision: 0.4218 - val_recall: 0.3602 Epoch 20/20 13/13 ━━━━━━━━━━━━━━━━━━━━ 33s 3s/step - accuracy: 0.9867 - auc: 0.9999 - loss: 0.0456 - precision: 0.9921 - recall: 0.9839 - val_accuracy: 0.4055 - val_auc: 0.7673 - val_loss: 3.8660 - val_precision: 0.4332 - val_recall: 0.3678 Training time: 659.30 seconds
In [20]:
print("\nValidation Accuracy Model3:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
print(f"{m}: {history3.history[m][-1]:.4f}")
Validation Accuracy Model3: val_accuracy: 0.4055 val_precision: 0.4332 val_recall: 0.3678 val_auc: 0.7673
In [21]:
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(history3.history['accuracy'], label='Train Accuracy')
plt.plot(history3.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(history3.history['loss'], label='Train Loss')
plt.plot(history3.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.tight_layout()
plt.show()
In [22]:
y_pred = model1a_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)
cm3 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm3, annot=True, fmt='d', cmap='Blues',
xticklabels=label_encoder.classes_,
yticklabels=label_encoder.classes_)
plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()
13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 125ms/step
Model1-B¶
In [23]:
np.random.seed(160)
tf.random.set_seed(160)
model1b_multi = Sequential()
model1b_multi.add(Conv2D(32, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model1b_multi.add(MaxPooling2D(pool_size=(2, 2)))
model1b_multi.add(Conv2D(64, (3, 3), activation='relu'))
model1b_multi.add(MaxPooling2D(pool_size=(2, 2)))
model1b_multi.add(Flatten())
model1b_multi.add(Dense(128, activation='relu'))
model1b_multi.add(Dropout(0.3))
model1b_multi.add(Dense(12, activation='softmax'))
model1b_multi.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc')
]
)
C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. super().__init__(activity_regularizer=activity_regularizer, **kwargs)
In [24]:
start_time = time.time()
history4= model1b_multi.fit(
X_train, y_train,
epochs=15,
batch_size=64,
validation_data=(X_test, y_test)
)
end_time = time.time()
time4 = end_time - start_time
print(f"Training time: {time4:.2f} seconds ({time4 / 60:.2f} minutes)")
Epoch 1/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 37s 1s/step - accuracy: 0.1688 - auc: 0.5577 - loss: 277.2318 - precision: 0.1606 - recall: 0.1203 - val_accuracy: 0.3652 - val_auc: 0.7518 - val_loss: 2.2955 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 2/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 34s 1s/step - accuracy: 0.3851 - auc: 0.7703 - loss: 2.1832 - precision: 0.5867 - recall: 0.0304 - val_accuracy: 0.3955 - val_auc: 0.7954 - val_loss: 2.0291 - val_precision: 0.5455 - val_recall: 0.0453 Epoch 3/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.5282 - auc: 0.8698 - loss: 1.6901 - precision: 0.7639 - recall: 0.1532 - val_accuracy: 0.4257 - val_auc: 0.8044 - val_loss: 1.9602 - val_precision: 0.5440 - val_recall: 0.1713 Epoch 4/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.6821 - auc: 0.9448 - loss: 1.1306 - precision: 0.8553 - recall: 0.4145 - val_accuracy: 0.3678 - val_auc: 0.8055 - val_loss: 2.2662 - val_precision: 0.4735 - val_recall: 0.2695 Epoch 5/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.7801 - auc: 0.9670 - loss: 0.8575 - precision: 0.8845 - recall: 0.5553 - val_accuracy: 0.4433 - val_auc: 0.8014 - val_loss: 2.3811 - val_precision: 0.5075 - val_recall: 0.3401 Epoch 6/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.8629 - auc: 0.9911 - loss: 0.4389 - precision: 0.9223 - recall: 0.7953 - val_accuracy: 0.4307 - val_auc: 0.7871 - val_loss: 2.8283 - val_precision: 0.4704 - val_recall: 0.3602 Epoch 7/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 35s 1s/step - accuracy: 0.9368 - auc: 0.9979 - loss: 0.2201 - precision: 0.9600 - recall: 0.8997 - val_accuracy: 0.4358 - val_auc: 0.7741 - val_loss: 3.3975 - val_precision: 0.4688 - val_recall: 0.3980 Epoch 8/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 35s 1s/step - accuracy: 0.9330 - auc: 0.9913 - loss: 0.3123 - precision: 0.9470 - recall: 0.9139 - val_accuracy: 0.4030 - val_auc: 0.7788 - val_loss: 2.8058 - val_precision: 0.4586 - val_recall: 0.3350 Epoch 9/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9369 - auc: 0.9922 - loss: 0.2830 - precision: 0.9527 - recall: 0.8920 - val_accuracy: 0.4559 - val_auc: 0.7755 - val_loss: 3.2140 - val_precision: 0.5061 - val_recall: 0.4207 Epoch 10/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9678 - auc: 0.9993 - loss: 0.1325 - precision: 0.9752 - recall: 0.9502 - val_accuracy: 0.4282 - val_auc: 0.7860 - val_loss: 3.2643 - val_precision: 0.4679 - val_recall: 0.3854 Epoch 11/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9704 - auc: 0.9995 - loss: 0.1152 - precision: 0.9789 - recall: 0.9520 - val_accuracy: 0.4660 - val_auc: 0.7932 - val_loss: 3.1306 - val_precision: 0.4882 - val_recall: 0.4156 Epoch 12/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9834 - auc: 0.9991 - loss: 0.0644 - precision: 0.9870 - recall: 0.9752 - val_accuracy: 0.4232 - val_auc: 0.7987 - val_loss: 3.3640 - val_precision: 0.4599 - val_recall: 0.3904 Epoch 13/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 34s 1s/step - accuracy: 0.9837 - auc: 0.9998 - loss: 0.0568 - precision: 0.9879 - recall: 0.9802 - val_accuracy: 0.4358 - val_auc: 0.7812 - val_loss: 3.7600 - val_precision: 0.4718 - val_recall: 0.4207 Epoch 14/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9933 - auc: 1.0000 - loss: 0.0276 - precision: 0.9953 - recall: 0.9916 - val_accuracy: 0.4332 - val_auc: 0.7918 - val_loss: 3.4574 - val_precision: 0.4687 - val_recall: 0.3955 Epoch 15/15 25/25 ━━━━━━━━━━━━━━━━━━━━ 33s 1s/step - accuracy: 0.9900 - auc: 0.9999 - loss: 0.0329 - precision: 0.9932 - recall: 0.9883 - val_accuracy: 0.4484 - val_auc: 0.7943 - val_loss: 3.4947 - val_precision: 0.4926 - val_recall: 0.4207 Training time: 506.91 seconds (8.45 minutes)
In [25]:
print("\nValidation Accuracy Model4:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
print(f"{m}: {history4.history[m][-1]:.4f}")
Validation Accuracy Model4: val_accuracy: 0.4484 val_precision: 0.4926 val_recall: 0.4207 val_auc: 0.7943
In [26]:
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(history4.history['accuracy'], label='Train Accuracy')
plt.plot(history4.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(history4.history['loss'], label='Train Loss')
plt.plot(history4.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.tight_layout()
plt.show()
In [27]:
y_pred = model1b_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)
cm4 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm4, annot=True, fmt='d', cmap='Blues',
xticklabels=label_encoder.classes_,
yticklabels=label_encoder.classes_)
plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()
13/13 ━━━━━━━━━━━━━━━━━━━━ 2s 121ms/step
Model2-A¶
In [28]:
np.random.seed(160)
tf.random.set_seed(160)
model2a_multi = Sequential()
model2a_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2a_multi.add(MaxPooling2D(pool_size=(2, 2)))
model2a_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2a_multi.add(MaxPooling2D(pool_size=(2, 2)))
model2a_multi.add(Flatten())
model2a_multi.add(Dense(64, activation='relu'))
model2a_multi.add(Dropout(0.4))
model2a_multi.add(Dense(12, activation='softmax'))
model2a_multi.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc')
]
)
model2a_multi.summary()
C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Model: "sequential_4"
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ ┃ Layer (type) ┃ Output Shape ┃ Param # ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ │ conv2d_8 (Conv2D) │ (None, 126, 515, 16) │ 160 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ max_pooling2d_8 (MaxPooling2D) │ (None, 63, 257, 16) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ conv2d_9 (Conv2D) │ (None, 61, 255, 32) │ 4,640 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ max_pooling2d_9 (MaxPooling2D) │ (None, 30, 127, 32) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ flatten_4 (Flatten) │ (None, 121920) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dense_8 (Dense) │ (None, 64) │ 7,802,944 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dropout_4 (Dropout) │ (None, 64) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dense_9 (Dense) │ (None, 12) │ 780 │ └─────────────────────────────────┴────────────────────────┴───────────────┘
Total params: 7,808,524 (29.79 MB)
Trainable params: 7,808,524 (29.79 MB)
Non-trainable params: 0 (0.00 B)
In [29]:
start_time = time.time()
history6= model2a_multi.fit(
X_train, y_train,
epochs=15,
batch_size=32,
validation_data=(X_test, y_test)
)
end_time = time.time()
time6 = end_time - start_time
print(f"Training time: {time6:.2f} seconds ({time6 / 60:.2f} minutes)")
Epoch 1/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 13s 218ms/step - accuracy: 0.1576 - auc: 0.5262 - loss: 89.9453 - precision: 0.1160 - recall: 0.0574 - val_accuracy: 0.2796 - val_auc: 0.6792 - val_loss: 2.3146 - val_precision: 0.0000e+00 - val_recall: 0.0000e+00 Epoch 2/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 211ms/step - accuracy: 0.2761 - auc: 0.6786 - loss: 2.3032 - precision: 0.6057 - recall: 0.0140 - val_accuracy: 0.3325 - val_auc: 0.7132 - val_loss: 2.1947 - val_precision: 0.5946 - val_recall: 0.0554 Epoch 3/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 210ms/step - accuracy: 0.3382 - auc: 0.7604 - loss: 2.0945 - precision: 0.7583 - recall: 0.0832 - val_accuracy: 0.3804 - val_auc: 0.7514 - val_loss: 2.1184 - val_precision: 0.7500 - val_recall: 0.0378 Epoch 4/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4139 - auc: 0.8156 - loss: 1.8752 - precision: 0.7625 - recall: 0.1354 - val_accuracy: 0.3804 - val_auc: 0.7733 - val_loss: 2.0503 - val_precision: 0.6207 - val_recall: 0.0453 Epoch 5/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4658 - auc: 0.8616 - loss: 1.6769 - precision: 0.8261 - recall: 0.2101 - val_accuracy: 0.4005 - val_auc: 0.7823 - val_loss: 2.0135 - val_precision: 0.6559 - val_recall: 0.1537 Epoch 6/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 213ms/step - accuracy: 0.4949 - auc: 0.9019 - loss: 1.4338 - precision: 0.8300 - recall: 0.3012 - val_accuracy: 0.4156 - val_auc: 0.7857 - val_loss: 2.0423 - val_precision: 0.6354 - val_recall: 0.1537 Epoch 7/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 212ms/step - accuracy: 0.5460 - auc: 0.9238 - loss: 1.2751 - precision: 0.8316 - recall: 0.3572 - val_accuracy: 0.4257 - val_auc: 0.7920 - val_loss: 2.0858 - val_precision: 0.6071 - val_recall: 0.2569 Epoch 8/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 211ms/step - accuracy: 0.5891 - auc: 0.9471 - loss: 1.0776 - precision: 0.8539 - recall: 0.4484 - val_accuracy: 0.4181 - val_auc: 0.7821 - val_loss: 2.2802 - val_precision: 0.5674 - val_recall: 0.3073 Epoch 9/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 216ms/step - accuracy: 0.6204 - auc: 0.9566 - loss: 0.9760 - precision: 0.8708 - recall: 0.4972 - val_accuracy: 0.4055 - val_auc: 0.7879 - val_loss: 2.2989 - val_precision: 0.5388 - val_recall: 0.2972 Epoch 10/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 214ms/step - accuracy: 0.6574 - auc: 0.9627 - loss: 0.8983 - precision: 0.8973 - recall: 0.5350 - val_accuracy: 0.4156 - val_auc: 0.7896 - val_loss: 2.2420 - val_precision: 0.5803 - val_recall: 0.2821 Epoch 11/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.6881 - auc: 0.9693 - loss: 0.8198 - precision: 0.8711 - recall: 0.5819 - val_accuracy: 0.4232 - val_auc: 0.7922 - val_loss: 2.4287 - val_precision: 0.5569 - val_recall: 0.3451 Epoch 12/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.7021 - auc: 0.9742 - loss: 0.7545 - precision: 0.8710 - recall: 0.6122 - val_accuracy: 0.4332 - val_auc: 0.7873 - val_loss: 2.4890 - val_precision: 0.5542 - val_recall: 0.3350 Epoch 13/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 214ms/step - accuracy: 0.7285 - auc: 0.9751 - loss: 0.7277 - precision: 0.8831 - recall: 0.6360 - val_accuracy: 0.4181 - val_auc: 0.7833 - val_loss: 2.4983 - val_precision: 0.5391 - val_recall: 0.3300 Epoch 14/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 219ms/step - accuracy: 0.7612 - auc: 0.9800 - loss: 0.6484 - precision: 0.8878 - recall: 0.6555 - val_accuracy: 0.4257 - val_auc: 0.7851 - val_loss: 2.5617 - val_precision: 0.5245 - val_recall: 0.3501 Epoch 15/15 50/50 ━━━━━━━━━━━━━━━━━━━━ 11s 215ms/step - accuracy: 0.7615 - auc: 0.9785 - loss: 0.6538 - precision: 0.8703 - recall: 0.6647 - val_accuracy: 0.4131 - val_auc: 0.7976 - val_loss: 2.3840 - val_precision: 0.5370 - val_recall: 0.2922 Training time: 162.97 seconds (2.72 minutes)
In [30]:
print("\nValidation Accuracy Model6:")
for m in ['val_accuracy', 'val_precision', 'val_recall', 'val_auc']:
print(f"{m}: {history6.history[m][-1]:.4f}")
Validation Accuracy Model6: val_accuracy: 0.4131 val_precision: 0.5370 val_recall: 0.2922 val_auc: 0.7976
In [31]:
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(history6.history['accuracy'], label='Train Accuracy')
plt.plot(history6.history['val_accuracy'], label='Val Accuracy')
plt.title('Model Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(history6.history['loss'], label='Train Loss')
plt.plot(history6.history['val_loss'], label='Val Loss')
plt.title('Model Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.tight_layout()
plt.show()
In [32]:
y_pred = model2a_multi.predict(X_test)
y_pred_classes = np.argmax(y_pred, axis=1)
y_true = np.argmax(y_test, axis=1)
cm6 = confusion_matrix(y_true, y_pred_classes)
plt.figure(figsize=(8, 6))
sns.heatmap(cm6, annot=True, fmt='d', cmap='Blues',
xticklabels=label_encoder.classes_,
yticklabels=label_encoder.classes_)
plt.xlabel('Predicted')
plt.ylabel('True')
plt.title('Confusion Matrix Heatmap')
plt.xticks(rotation=90)
plt.yticks(rotation=0)
plt.tight_layout()
plt.show()
13/13 ━━━━━━━━━━━━━━━━━━━━ 1s 51ms/step
In [33]:
np.random.seed(160)
tf.random.set_seed(160)
model2b_multi = Sequential()
model2b_multi.add(Conv2D(16, (3, 3), activation='relu', input_shape=(128, 517, 1)))
model2b_multi.add(MaxPooling2D(pool_size=(2, 2)))
model2b_multi.add(Conv2D(32, (3, 3), activation='relu'))
model2b_multi.add(MaxPooling2D(pool_size=(2, 2)))
model2b_multi.add(Flatten())
model2b_multi.add(Dense(64, activation='relu'))
model2b_multi.add(Dropout(0.4))
model2b_multi.add(Dense(12, activation='softmax'))
model2b_multi.compile(
optimizer='adam',
loss='categorical_crossentropy',
metrics=[
'accuracy',
Precision(name='precision'),
Recall(name='recall'),
AUC(name='auc')
]
)
model2b_multi.summary()
C:\Users\Nidhi\anaconda3\envs\py311\Lib\site-packages\keras\src\layers\convolutional\base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Model: "sequential_5"
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ ┃ Layer (type) ┃ Output Shape ┃ Param # ┃ ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ │ conv2d_10 (Conv2D) │ (None, 126, 515, 16) │ 160 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ max_pooling2d_10 (MaxPooling2D) │ (None, 63, 257, 16) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ conv2d_11 (Conv2D) │ (None, 61, 255, 32) │ 4,640 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ max_pooling2d_11 (MaxPooling2D) │ (None, 30, 127, 32) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ flatten_5 (Flatten) │ (None, 121920) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dense_10 (Dense) │ (None, 64) │ 7,802,944 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dropout_5 (Dropout) │ (None, 64) │ 0 │ ├─────────────────────────────────┼────────────────────────┼───────────────┤ │ dense_11 (Dense) │ (None, 12) │ 780 │ └─────────────────────────────────┴────────────────────────┴───────────────┘
Total params: 7,808,524 (29.79 MB)
Trainable params: 7,808,524 (29.79 MB)
Non-trainable params: 0 (0.00 B)
In [46]:
# saving bestmodel out of all
model1b_multi.save("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel1.h5")
WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.
In [47]:
model2_multi.save("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel2.h5")
WARNING:absl:You are saving your model as an HDF5 file via `model.save()` or `keras.saving.save_model(model)`. This file format is considered legacy. We recommend using instead the native Keras format, e.g. `model.save('my_model.keras')` or `keras.saving.save_model(model, 'my_model.keras')`.
External Test Data¶
In [48]:
# Loading required libraries
import librosa
import librosa.display
import os
In [61]:
import librosa
import numpy as np
import tensorflow as tf
import os
from collections import Counter
SAMPLE_RATE = 32000
WINDOW_SIZE = 10
STEP_SIZE = 1.5
N_MELS = 128
HOP_LENGTH = 128
WIN_LENGTH = 512
EXPECTED_WIDTH = 517
# ─────── Labels & Models ─────────────────────────────────────────────────────
species_labels = species # your list of 12 species names
model1 = tf.keras.models.load_model("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel1.h5")
model2 = tf.keras.models.load_model("C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\bestmodel2.h5")
models = {"Model 1": model1, "Model 2": model2}
mp3_files = [
"C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test1.mp3",
"C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test2.mp3",
"C:\\Users\\Nidhi\\Documents\\College Material\\Spring Quarter 2025\\SML2\Homework3\\Homework 3_test_birds_test3.mp3",
]
def sliding_window_preds(audio, sr, model):
window_len = int(sr * WINDOW_SIZE)
step_len = int(sr * STEP_SIZE)
num_windows = max(1, int((len(audio) - window_len) / step_len) + 1)
all_preds = []
for i in range(num_windows):
start, end = i*step_len, i*step_len + window_len
y_win = audio[start:end].copy()
# spectrogram
mel = librosa.feature.melspectrogram(
y=y_win, sr=sr,
n_mels=N_MELS,
hop_length=HOP_LENGTH,
win_length=WIN_LENGTH
)
mel_db = librosa.power_to_db(mel, ref=np.max)
norm = (mel_db - mel_db.min()) / (mel_db.max() - mel_db.min())
# resize to EXPECTED_WIDTH
if norm.shape[1] > EXPECTED_WIDTH:
norm = norm[:, :EXPECTED_WIDTH]
else:
pad_width = EXPECTED_WIDTH - norm.shape[1]
norm = np.pad(norm, ((0,0),(0,pad_width)), mode='constant')
# predict
inp = np.expand_dims(norm, axis=(0,-1)) # shape: (1,128,517,1)
pred = model.predict(inp, verbose=0)[0] # vector of 12 probs
all_preds.append(pred)
return np.stack(all_preds) # shape: (num_windows, 12)
WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model. WARNING:absl:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
In [62]:
for model_idx, model in enumerate([model1, model2], start=1):
print(f"\n============================")
print(f"Predictions using Model {model_idx}")
print(f"============================")
for file in mp3_files:
print(f"\nFile: {os.path.basename(file)}")
y, sr = librosa.load(file, sr=SAMPLE_RATE)
preds = sliding_window_preds(y, sr, model)
# Average and top predictions
avg_pred = np.mean(preds, axis=0)
top3_avg = np.argsort(avg_pred)[::-1][:3]
print("Top 3 Average Predictions:")
for i in top3_avg:
print(f" - {species_labels[i]}: {avg_pred[i]:.2f}")
# Count most frequent top-1 prediction per window
top1_per_window = np.argmax(preds, axis=1)
counts = Counter(top1_per_window)
print("\nTop species per window:")
for idx, cnt in counts.most_common():
print(f" - {species_labels[idx]}: {cnt} windows")
# Plot the segment with the strongest prediction
best_idx = np.argmax(np.max(preds, axis=1))
best_start = int(best_idx * SAMPLE_RATE * STEP_SIZE)
best_end = best_start + int(SAMPLE_RATE * WINDOW_SIZE)
y_best = y[best_start:best_end]
mel = librosa.feature.melspectrogram(y=y_best, sr=sr, n_mels=N_MELS,
hop_length=HOP_LENGTH, win_length=WIN_LENGTH)
mel_db = librosa.power_to_db(mel, ref=np.max)
plt.figure(figsize=(10, 4))
librosa.display.specshow(mel_db, sr=sr, x_axis='time', y_axis='mel',
cmap='gray_r', hop_length=HOP_LENGTH)
plt.colorbar(format='%+2.0f dB')
plt.title(f"Model {model_idx} - Strongest Segment - {os.path.basename(file)}")
plt.tight_layout()
plt.show()
============================ Predictions using Model 1 ============================ File: Homework 3_test_birds_test1.mp3 Top 3 Average Predictions: - rewbla: 0.84 - sonspa: 0.07 - daejun: 0.06 Top species per window: - rewbla: 9 windows
File: Homework 3_test_birds_test2.mp3 Top 3 Average Predictions: - rewbla: 0.59 - daejun: 0.16 - sonspa: 0.14 Top species per window: - rewbla: 1 windows
File: Homework 3_test_birds_test3.mp3 Top 3 Average Predictions: - rewbla: 0.92 - daejun: 0.04 - sonspa: 0.03 Top species per window: - rewbla: 4 windows
============================ Predictions using Model 2 ============================ File: Homework 3_test_birds_test1.mp3 Top 3 Average Predictions: - houspa: 0.45 - sonspa: 0.28 - houfin: 0.27 Top species per window: - houspa: 9 windows
File: Homework 3_test_birds_test2.mp3 Top 3 Average Predictions: - houspa: 0.42 - sonspa: 0.29 - houfin: 0.28 Top species per window: - houspa: 1 windows
File: Homework 3_test_birds_test3.mp3 Top 3 Average Predictions: - houspa: 0.48 - sonspa: 0.27 - houfin: 0.25 Top species per window: - houspa: 4 windows
In [ ]: